Fully Convolutional Network

In [1]:
# Imports
import os
from keras.layers import *
from keras.models import Model, load_model
from keras.regularizers import l2
from keras.callbacks import ModelCheckpoint
from keras.optimizers import RMSprop, Adam, SGD
import matplotlib.pyplot as plt
import scipy.misc as m
Using TensorFlow backend.

FCN model

In [2]:
def FCN32(d=0.2):
    inp = Input(shape=(None,None,3))
    
    # Block 1
    c11 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1', kernel_regularizer=l2(5e-4))(inp)
    c12 = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2', kernel_regularizer=l2(5e-4))(c11)
    p1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(c12)

    # Block 2
    c21 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1', kernel_regularizer=l2(5e-4))(p1)
    c22 = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2', kernel_regularizer=l2(5e-4))(c21)
    p2 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(c22)

# #     # Block 3
#     c31 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1', kernel_regularizer=l2(5e-4))(p2)
#     c32 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2', kernel_regularizer=l2(5e-4))(c31)
#     c33 = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3', kernel_regularizer=l2(5e-4))(c32)
#     p3 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(c33)

#     # Block 4
#     c41 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1', kernel_regularizer=l2(5e-4))(p3)
#     c42 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2', kernel_regularizer=l2(5e-4))(c41)
#     c43 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3', kernel_regularizer=l2(5e-4))(c42)
#     p4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(c43)

#     # Block 5
#     c51 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1', kernel_regularizer=l2(5e-4))(p4)
#     c52 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2', kernel_regularizer=l2(5e-4))(c51)
#     c53 = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3', kernel_regularizer=l2(5e-4))(c52)
#     p5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(c53)

    # Convolutional layers transfered from fully-connected layers
#     fc1 = Conv2D(1024, (7, 7), activation='relu', padding='same', name='fc1')(p5)
#     fc1 = Dropout(d)(fc1)
#     fc2 = Conv2D(1024, (1, 1), activation='relu', padding='same', name='fc2')(fc1)
#     fc2 = Dropout(d)(fc2)
    #classifying layer
#     out = Conv2D(2, (1, 1), kernel_initializer='he_normal', activation='linear', padding='valid', strides=(1, 1))(p2)

#     output = Conv2DTranspose(2, (32,32), activation='softmax', strides=(32, 32))(out)
    output = UpSampling2D(size=(4,4))(p2)
    output = Conv2D(2,(1,1), activation='softmax')(output)
    
    model = Model(inputs=inp, outputs=output)
    
    return model
In [3]:
fcn32s = FCN32()
fcn32s.summary()
WARNING:tensorflow:From /home/augustocms/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_1 (InputLayer)         (None, None, None, 3)     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, None, None, 64)    1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, None, None, 64)    36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, None, None, 64)    0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, None, None, 128)   73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, None, None, 128)   147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, None, None, 128)   0         
_________________________________________________________________
up_sampling2d_1 (UpSampling2 (None, None, None, 128)   0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, None, None, 2)     258       
=================================================================
Total params: 260,418
Trainable params: 260,418
Non-trainable params: 0
_________________________________________________________________

DRIVE

Read training and validation sets

In [4]:
dir_drive = "/home/augustocms/datasets/drive/training/"
dir_img = dir_drive + "images/"
dir_seg = dir_drive + "1st_manual/"
dir_test = "/home/augustocms/datasets/drive/test/"
dir_test_img = dir_test + "images/"
dir_test_seg = dir_test + "1st_manual/"

with open(dir_drive + 'train.txt') as f:
    x = f.read().splitlines()
imgs = np.array(x)

with open(dir_drive + 'val.txt') as f:
    x = f.read().splitlines()
val_imgs = np.array(x)

with open(dir_test + 'test.txt') as f:
    x = f.read().splitlines()
test_imgs = np.array(x)

print(imgs[0])
21_training.tif 21_manual1.gif 21_training_mask.gif

Load data

In [5]:
def extract_patches(img, size):
    H, W, *c = img.shape
    h = H//size[0]
    w = W//size[1]
    patches = []
    for i in range(h):
        for j in range(w):
            patches.append(img[i*size[0]:(i+1)*size[0], j*size[1]:(j+1)*size[1]])
    return np.array(patches)

def to_categorical(patches):
    n, H, W = patches.shape
    segmentation = np.zeros((n,H,W,2))
    for i in range(n):
        segmentation[i,:,:,0] = (patches[i,:,:] == 0).astype(int)
        segmentation[i,:,:,1] = (patches[i,:,:] > 0).astype(int)
    return segmentation
In [6]:
images = []
gt = []
for line in imgs:
    names = line.split(' ')
    images.append(m.imread(dir_img + names[0]))
    gt.append(m.imread(dir_seg + names[1]))
images = np.array(images)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  """
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  
In [7]:
patch_size = 32

X_train = []
y_train = []
for i, img in enumerate(images):
    seg_img = gt[i]
    X_train.append(extract_patches(img, (patch_size, patch_size)))
    y_patches = to_categorical(extract_patches(seg_img, (patch_size,patch_size)))
    y_train.append(y_patches)
    
    fig = plt.figure(figsize=(20,40))
    ax = fig.add_subplot(1,2,1)
    ax.imshow(img)
    ax.set_title("original image")
    
    ax = fig.add_subplot(1,2,2)
    ax.imshow(seg_img, cmap='gray')
    ax.set_title('ground truth')
    
    plt.show()
In [8]:
X_train = np.array(X_train)
X_train = np.reshape(X_train, (-1, patch_size,patch_size,3))
y_train = np.array(y_train)
y_train = np.reshape(y_train, (-1, patch_size, patch_size,2))
print(X_train.shape, y_train.shape)
(6120, 32, 32, 3) (6120, 32, 32, 2)
In [9]:
X_val = []
y_val = []
for line in val_imgs:
    names = line.split(' ')
    val_img = m.imread(dir_img + names[0])
    val_seg = m.imread(dir_seg + names[1])
    X_val.append(extract_patches(val_img, (patch_size, patch_size)))
    y_patches = to_categorical(extract_patches(val_seg, (patch_size,patch_size)))
    y_val.append(y_patches)
    
X_val, y_val = np.array(X_val), np.array(y_val)
X_val = np.reshape(X_val, (-1, patch_size, patch_size, 3))
y_val = np.reshape(y_val, (-1, patch_size, patch_size, 2))
print(X_val.shape, y_val.shape)
(1224, 32, 32, 3) (1224, 32, 32, 2)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  """
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  

Checking if patch and ground truth matches:

In [10]:
n = 100
fig = plt.figure(figsize=(5,10))
ax = fig.add_subplot(1,2,1)
ax.imshow(X_train[n][:,:])
ax.set_title('img')

ax = fig.add_subplot(1,2,2)
ax.imshow(y_train[n][:,:,1], cmap='gray')
ax.set_title('gt')

plt.show()

Training

In [11]:
opt = Adam(lr=1e-4)

fcn32s.compile(loss='binary_crossentropy',
             optimizer=opt,
             metrics=['accuracy'])

checkpoint = ModelCheckpoint('weights/fcn_drive.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
In [12]:
# fcn32s.fit(X_train, y_train,
#           validation_data=(X_val, y_val),
#           epochs=60, batch_size=8,
#           verbose=2,
#           callbacks=[checkpoint])

# fcn32s.save('models/fcn_drive.h5')

fcn32s = load_model('models/fcn_drive.h5')
WARNING:tensorflow:From /home/augustocms/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
WARNING:tensorflow:From /home/augustocms/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.

Predictions on test set

In [13]:
X_test = []
y_test = []
for line in test_imgs:
    names = line.split(' ')
    test_img = m.imread(dir_test_img + names[0])
    test_seg = m.imread(dir_test_seg + names[1])
    X_test.append(test_img)
    y_ing = to_categorical(np.array([test_seg]))
    y_test.append(y_ing)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  """
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  
In [14]:
y_test = np.array(y_test)
y_test = np.squeeze(y_test)
X_test = np.array(X_test)
print(X_test.shape, y_test.shape)
(20, 584, 565, 3) (20, 584, 565, 2)
In [15]:
y_pred = fcn32s.predict(X_test)
y_predi = np.argmax(y_pred, axis=3)
y_testi = np.argmax(y_test, axis=3)
print(y_testi.shape,y_predi.shape)
(20, 584, 565) (20, 584, 564)
In [16]:
gtp = (y_testi == 1).astype(int)[:,:,:-1]
pp = (y_predi == 1).astype(int)
gtn = (y_testi == 0).astype(int)[:,:,:-1]
pn = (y_predi == 0).astype(int)

TP = (gtp*pp).sum()
TN = (gtn*pn).sum()
FP = (gtn*pp).sum()
FN = (gtp*pn).sum()

Precision = TP/(TP+FP)
Sensitivity = TP/(TP+FN)
FMeasure = 2 * (Precision*Sensitivity)/(Precision + Sensitivity)
print(Precision, Sensitivity, FMeasure)
0.737615168349 0.602849752139
In [17]:
shape = (584, 565)
n_classes= 1

for i in range(10):
    img_is  = X_test[i]
    seg = y_predi[i]
    segtest = y_testi[i]

    fig = plt.figure(figsize=(20,40))    
    ax = fig.add_subplot(1,3,1)
    ax.imshow(img_is, cmap='gray')
    ax.set_title("original")
    
    ax = fig.add_subplot(1,3,2)
    ax.imshow(seg, cmap='gray')
    ax.set_title("predicted class")
    
    ax = fig.add_subplot(1,3,3)
    ax.imshow(segtest, cmap='gray')
    ax.set_title("true class")
    plt.show()
    
    m.imsave('images/drive/'+str(i)+'_orig.png', img_is)
    m.imsave('images/drive/'+str(i)+'_pred_fcn.png', seg)
    m.imsave('images/drive/'+str(i)+'_segm.png', segtest)
    
    print(img_is.shape, seg.shape, segtest.shape)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:23: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:24: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:25: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)
(584, 565, 3) (584, 564) (584, 565)

DIBCO

Read training and validation sets

In [18]:
dir_dibco = "/home/augustocms/scratch/DIBCO/"

with open(dir_dibco + 'train.txt') as f:
    x = f.read().splitlines()
imgs = np.array(x)

with open(dir_dibco + 'val.txt') as f:
    x = f.read().splitlines()
val_imgs = np.array(x)

with open(dir_dibco + 'test.txt') as f:
    x = f.read().splitlines()
test_imgs = np.array(x)

print(imgs[0])
2009/H01.bmp 2009/H01.tiff

Load data

In [19]:
images = []
gt = []
for line in imgs:
    names = line.split()
    images.append(m.imread(dir_dibco + names[0], mode='RGB'))
    gt.append(m.imread(dir_dibco + names[1], mode='L'))
images = np.array(images)
gt = np.array(gt)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  """
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  
In [20]:
patch_size = 32

X_train = []
y_train = []
for i, img in enumerate(images):
    seg_img = gt[i]
    X_train.append(extract_patches(img, (patch_size, patch_size)))
    y_patches = to_categorical(extract_patches(seg_img, (patch_size, patch_size)))
    y_train.append(y_patches)
    
    fig = plt.figure(figsize=(20,40))
    ax = fig.add_subplot(1,2,1)
    ax.imshow(img)
    ax.set_title("original image")
    
    ax = fig.add_subplot(1,2,2)
    ax.imshow(seg_img, cmap='gray')
    ax.set_title('ground truth')
    
    plt.show()
In [21]:
X_train = np.array(X_train)
X_train = np.vstack(X_train)
X_train = np.reshape(X_train, (-1, patch_size,patch_size,3))
y_train = np.array(y_train)
y_train = np.vstack(y_train)
y_train = np.reshape(y_train, (-1, patch_size, patch_size,2))
print(X_train.shape, y_train.shape)
(90210, 32, 32, 3) (90210, 32, 32, 2)
In [22]:
X_val = []
y_val = []
for line in val_imgs:
    names = line.split()
    val_img = m.imread(dir_dibco + names[0], mode='RGB')
    val_seg = m.imread(dir_dibco + names[1], mode='L')
    X_val.append(extract_patches(val_img, (patch_size, patch_size)))
    y_patches = to_categorical(extract_patches(val_seg, (patch_size,patch_size)))
    y_val.append(y_patches)
    
X_val, y_val = np.array(X_val), np.array(y_val)
X_val = np.vstack(X_val)
X_val = np.reshape(X_val, (-1, patch_size, patch_size, 3))
y_val = np.vstack(y_val)
y_val = np.reshape(y_val, (-1, patch_size, patch_size, 2))
print(X_val.shape, y_val.shape)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:5: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  """
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  
(31011, 32, 32, 3) (31011, 32, 32, 2)

Checking if patch and ground truth matches:

In [23]:
n = 533
fig = plt.figure(figsize=(5,10))
ax = fig.add_subplot(1,2,1)
ax.imshow(X_train[n][:,:])
ax.set_title('img')

ax = fig.add_subplot(1,2,2)
ax.imshow(y_train[n][:,:,1], cmap='gray')
ax.set_title('gt')

plt.show()

Training

In [24]:
opt = Adam(lr=1e-4)

fcn32s.compile(loss='binary_crossentropy',
             optimizer=opt,
             metrics=['accuracy'])

checkpoint = ModelCheckpoint('weights/fcn_dibco.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min')
In [25]:
# fcn32s.fit(X_train, y_train,
#           validation_data=(X_val, y_val),
#           epochs=60, batch_size=32,
#           verbose=2,
#           callbacks=[checkpoint])

# fcn32s.save('models/fcn_dibco.h5')

fcn32s = load_model('models/fcn_dibco.h5')

Predictions on test set

In [26]:
def crop(img, mult):
    H, W, *c = img.shape
    h = H - H%mult
    w = W - W%mult
    return img[:h,:w]
    
X_test = []
y_test = []
for line in test_imgs:
    names = line.split()
    test_img = m.imread(dir_dibco + names[0], mode='RGB')
    test_seg = m.imread(dir_dibco + names[1], mode='L')
    X_test.append(crop(test_img, 4))
    y_ing = to_categorical(np.array([test_seg]))
    y_test.append(crop(y_ing[0], 4))
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:11: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  # This is added back by InteractiveShellApp.init_path()
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:12: DeprecationWarning: `imread` is deprecated!
`imread` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imread`` instead.
  if sys.path[0] == '':
In [27]:
y_predi = []
y_testi = []
for i, ing in enumerate(X_test):
    y_pred = fcn32s.predict(np.asarray([ing]))
    y_predi.append(np.argmax(y_pred, axis=3))
    y_testi.append(np.argmax(y_test[i], axis=2))
In [28]:
for i, ing in enumerate(y_predi):
    gtn = (y_testi[i] == 1).astype(int)
    pn = (ing[0] == 1).astype(int)
    gtp = (y_testi[i] == 0).astype(int)
    pp = (ing[0] == 0).astype(int)

TP = (gtp*pp).sum()
TN = (gtn*pn).sum()
FP = (gtn*pp).sum()
FN = (gtp*pn).sum()

Precision = TP/(TP+FP)
Sensitivity = TP/(TP+FN)
print(Precision, Sensitivity)
0.362673213349 0.886961621549
In [29]:
# Calculate other metrics as described by DIBCO competition
In [30]:
for i in range(10):
    img_is  = X_test[i]
    seg = y_predi[i][0]
    segtest = y_testi[i]

    fig = plt.figure(figsize=(20,40))    
    ax = fig.add_subplot(1,3,1)
    ax.imshow(img_is, cmap='gray')
    ax.set_title("original")
    
    ax = fig.add_subplot(1,3,2)
    ax.imshow(seg, cmap='gray')
    ax.set_title("predicted class")
    
    ax = fig.add_subplot(1,3,3)
    ax.imshow(segtest, cmap='gray')
    ax.set_title("true class")
    plt.show()
    
    m.imsave('images/dibco/'+str(i)+'_orig.png', img_is)
    m.imsave('images/dibco/'+str(i)+'_pred_fcn.png', seg)
    m.imsave('images/dibco/'+str(i)+'_segm.png', segtest)
    
    print(img_is.shape, seg.shape, segtest.shape)
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:20: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:21: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
/home/augustocms/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:22: DeprecationWarning: `imsave` is deprecated!
`imsave` is deprecated in SciPy 1.0.0, and will be removed in 1.2.0.
Use ``imageio.imwrite`` instead.
(432, 1776, 3) (432, 1776) (432, 1776)
(808, 1504, 3) (808, 1504) (808, 1504)
(628, 1972, 3) (628, 1972) (628, 1972)
(508, 1012, 3) (508, 1012) (508, 1012)
(288, 1504, 3) (288, 1504) (288, 1504)
(888, 1680, 3) (888, 1680) (888, 1680)
(580, 2544, 3) (580, 2544) (580, 2544)
(920, 3932, 3) (920, 3932) (920, 3932)
(284, 1212, 3) (284, 1212) (284, 1212)
(960, 3216, 3) (960, 3216) (960, 3216)